{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# week8--复现Densely Connected Convolutional Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.补充densenet实现代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Contains a variant of the densenet model definition.\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "slim = tf.contrib.slim\n",
    "\n",
    "# 高斯分布初始化\n",
    "def trunc_normal(stddev): \n",
    "    return tf.truncated_normal_initializer(stddev=stddev)\n",
    "\n",
    "# 定义复合函数作为一个非线性变换的小模块 BN（批归一化） - RELU（激活） - conv（卷积） - dropout（随机失活）\n",
    "# 应用于论文提到的 bottlneck 和 其它有卷积操作的地方\n",
    "# 参数： current - 当前的输出，num_outputs - 输出通道数， kernel_size - 卷积核大小\n",
    "def bn_act_conv_drp(current, num_outputs, kernel_size, scope='block'):\n",
    "    current = slim.batch_norm(current, scope=scope + '_bn')\n",
    "    current = tf.nn.relu(current)\n",
    "    current = slim.conv2d(current, num_outputs, kernel_size, scope=scope + '_conv') # 默认参数 padding = \"same\"\n",
    "    current = slim.dropout(current, scope=scope + '_dropout')\n",
    "    return current\n",
    "\n",
    "# densenet的模块组成，连续进行layers次的bottleneck和复合函数变化，每次得出的结果与上一次的结果拼接，作为下一次变换的输入\n",
    "# 返回该block的最终输出 net\n",
    "# 参数：net - 当前的输出，layers - block中重复一个小模块的次数， growth - 增长率单位（论文中的k）每次输出的通道数\n",
    "# block最终输出的net的通道数为 growth0 + growth * layers\n",
    "# 虽然每次循环都会增加上次的通道数，但bottleneck对每次的输入都限制了通道数的增长\n",
    "def block(net, layers, growth, scope='block'):\n",
    "    for idx in range(layers):\n",
    "        bottleneck = bn_act_conv_drp(net, 4 * growth, [1, 1],\n",
    "                                     scope=scope + '_conv1x1' + str(idx))\n",
    "        tmp = bn_act_conv_drp(bottleneck, growth, [3, 3],\n",
    "                              scope=scope + '_conv3x3' + str(idx))\n",
    "        net = tf.concat(axis=3, values=[net, tmp])\n",
    "    return net\n",
    "\n",
    "\n",
    "def densenet(images, num_classes=1001, is_training=False,\n",
    "             dropout_keep_prob=0.8,\n",
    "             scope='densenet'):\n",
    "    \"\"\"Creates a variant of the densenet model.\n",
    "\n",
    "      images: A batch of `Tensors` of size [batch_size, height, width, channels].\n",
    "      num_classes: the number of classes in the dataset.\n",
    "      is_training: specifies whether or not we're currently training the model.\n",
    "        This variable will determine the behaviour of the dropout layer.\n",
    "      dropout_keep_prob: the percentage of activation values that are retained.\n",
    "      prediction_fn: a function to get predictions out of logits.\n",
    "      scope: Optional variable_scope.\n",
    "\n",
    "    Returns:\n",
    "      logits: the pre-softmax activations, a tensor of size\n",
    "        [batch_size, `num_classes`]\n",
    "      end_points: a dictionary from components of the network to the corresponding\n",
    "        activation.\n",
    "    \"\"\"\n",
    "    growth = 24             # growth rate，对应于文章中的k，限制网络的宽度\n",
    "    compression_rate = 0.5  # 压缩率，用于在transition layers压缩feature map的通道数\n",
    "\n",
    "    def reduce_dim(input_feature):\n",
    "        return int(int(input_feature.shape[-1]) * compression_rate)\n",
    "\n",
    "    end_points = {}\n",
    "\n",
    "    with tf.variable_scope(scope, 'DenseNet', [images, num_classes]):\n",
    "        with slim.arg_scope(bn_drp_scope(is_training=is_training,\n",
    "                                         keep_prob=dropout_keep_prob)) as ssc:\n",
    "            # 原始图像大小为224*224*3\n",
    "            # 初始化卷积操作，输出feature map 为 112*112*48\n",
    "            end_point = 'Pre_Conv2d_7x7'\n",
    "            current = slim.conv2d(images, 2*growth, [7, 7], stride=2, padding='same', scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "            # 最大池化，降空间尺寸， 输出feature map 为 56*56*48\n",
    "            end_point = 'Pre_Pool_3x3'\n",
    "            current = slim.max_pool2d(current, [3, 3], stride=2, padding='same', scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "\n",
    "            # DenseBlock1 输出feature map 为 56*56\n",
    "            end_point = 'Block1'\n",
    "            current = block(current, 6, growth, scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "            # Transition1 输出feature map 为 28*28\n",
    "            end_point = 'Trans1_Conv2d_1x1'\n",
    "            current = bn_act_conv_drp(current, growth, [1, 1], scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "            end_point = 'Trans1_avgPool_2x2'\n",
    "            current = slim.avg_pool2d(current, [2, 2], stride=2, padding='same', scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "\n",
    "            # DenseBlock2 输出feature map 为 28*28\n",
    "            end_point = 'Block2'\n",
    "            current =  block(current, 12, growth, scope=end_point) \n",
    "            end_points[end_point] = current\n",
    "            # Transition2 输出feature map 为 14*14\n",
    "            end_point = 'Trans2_Conv2d_1x1'                     \n",
    "            current = bn_act_conv_drp(current, growth, [1, 1], scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "            end_point = 'Trans2_avgPool_2x2' \n",
    "            current = slim.avg_pool2d(current, [2, 2], stride=2, padding='same', scope=end_point) \n",
    "            end_points[end_point] = current\n",
    "\n",
    "            # DenseBlock3 输出feature map 为 14*14\n",
    "            end_point = 'Block3'\n",
    "            current =  block(current, 24, growth, scope=end_point) \n",
    "            end_points[end_point] = current\n",
    "            # Transition3 输出feature map 为 7*7\n",
    "            end_point = 'Trans3_Conv2d_1x1'\n",
    "            current = bn_act_conv_drp(current, growth, [1, 1], scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "            end_point = 'Trans3_avgPool_2x2' \n",
    "            current = slim.avg_pool2d(current, [2, 2], stride=2, padding='same', scope=end_point) \n",
    "            end_points[end_point] = current\n",
    "\n",
    "            # DenseBlock4 输出feature map 为 7*7\n",
    "            end_point = 'Block4'\n",
    "            current =  block(current, 16, growth, scope=end_point) \n",
    "            end_points[end_point] = current\n",
    "            # 全局池化 kernel size 7*7\n",
    "            end_point = 'Global_Pool_7x7' \n",
    "            current = slim.avg_pool2d(current, [7, 7], padding='same', scope=end_point)  \n",
    "            end_points[end_point] = current\n",
    "\n",
    "            # flatten后做全连接\n",
    "            end_point = 'Pre_Flatten'\n",
    "            current =  slim.flatten(current, scope=end_point)\n",
    "            end_points[end_point] = current\n",
    "            end_point = 'FC_Logits'\n",
    "            logits = slim.fully_connected(current, num_classes, activation_fn=None, scope=end_point)\n",
    "            end_points[end_point] = logits\n",
    "\n",
    "            # softmax分类\n",
    "            end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions')\n",
    "            \n",
    "    return logits, end_points\n",
    "\n",
    "def bn_drp_scope(is_training=True, keep_prob=0.8):\n",
    "    keep_prob = keep_prob if is_training else 1\n",
    "    with slim.arg_scope(\n",
    "        [slim.batch_norm],\n",
    "            scale=True, is_training=is_training, updates_collections=None):\n",
    "        with slim.arg_scope(\n",
    "            [slim.dropout],\n",
    "                is_training=is_training, keep_prob=keep_prob) as bsc:\n",
    "            return bsc\n",
    "\n",
    "\n",
    "def densenet_arg_scope(weight_decay=0.004):\n",
    "    \"\"\"Defines the default densenet argument scope.\n",
    "\n",
    "    Args:\n",
    "      weight_decay: The weight decay to use for regularizing the model.\n",
    "\n",
    "    Returns:\n",
    "      An `arg_scope` to use for the inception v3 model.\n",
    "    \"\"\"\n",
    "    with slim.arg_scope(\n",
    "        [slim.conv2d],\n",
    "        weights_initializer=tf.contrib.layers.variance_scaling_initializer(\n",
    "            factor=2.0, mode='FAN_IN', uniform=False),\n",
    "        activation_fn=None, biases_initializer=None, padding='same',\n",
    "            stride=1) as sc:\n",
    "        return sc\n",
    "\n",
    "\n",
    "densenet.default_image_size = 224"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. DenseNet网络分析与growth参数理解"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1）densenet实现细节分析：  \n",
    "----输入图片为224x224x3，首先使用 7x7 卷积核，2 步长进行卷积，将图片变为 112x112 大小，再使用 3x3核，2 步长的最大池化，得到 56x56 的feature map，此时通道数为2*growth，即48  \n",
    "\n",
    "----接下来就进行4个dense block的变换（特征抽取过程），每个block都使用相同的1x1 的bottleneck layer和 3x3的卷积核进行多次重复操作，每个block中的重复次数不一，每个小模块的输入都是该block内部前面所有输入的拼接（xl= H([x0,x1,...xl-1])），为了保证能正常拼接，同一个block中的feature map空间尺寸保持不变，但通道数增加。为了限制通道数的快速增加，每个小模块进行3x3卷积前增加了1x1的bottleneck layers，既能降维减少计算量又能融合各个通道。  \n",
    "\n",
    "----每个block之间通过transition layers进行连接，整个网络的feature map空间尺寸在这些transition layers中依次减小。每个block最终输出的feature map通道数为growth0 + growth * layers，虽然内部有bottleneck进行通道数的限制，但整个block最终输出的feature map通道数依旧很大，transition layers通过1x1的卷积对通道数进行降维（融合了每个block输出的各个通道），再通过2x2的平均池化对空间尺寸进行降维，使输入到下一个block的feature map达到一个合理的特征尺寸  \n",
    "\n",
    "----在最后一个block输出时，使用7x7的全局平均池化，将整个feature map的尺寸变成一个长条形的 1x1xN 的特征向量，并在全连接后用于softmax进行分类  \n",
    "\n",
    "----论文中所涉及到的卷积，实际上是一个复合操作，包含了 BN（批归一化） - RELU（激活） - conv（卷积） - dropout（随机失活）\n",
    "\n",
    "\n",
    "2）growth rate  \n",
    "论文中提出的参数growth rate（增长率K）关联与整个densenet的中每个阶段的feature map通道数的变化（K的值是个超参数，本代码中为24），通道数的变化以K为一个单位，进行整数倍的增加，应用于所有进行通道数变换的操作，如最开始的初始化卷积操作使用2*growth，block内部的bottleneck layers使用4*growth，block内部的3x3卷积使用1*growth，而transition layer中的卷积操作使用1*growth。 growth控制着整个网络的宽度（避免通道数增长太快），选择小的growth可以让网络变窄，计算的参数数量也会减少。  \n",
    "\n",
    "3）densenet网络的优势  \n",
    "resnet：xl=H(xl-1)+(xl-1)  \n",
    "densenet：xl= H([x0,x1,...xl-1])  \n",
    "相对于resnet来说，densenet使用前方更多的feature map，在前向传播的时候能更充分的抽取特征，加强了feature的传递，在反向传播的时候也能减缓梯度消失的问题，也正因为如此，对于densenet来说，网络的深度限制会放宽。  \n",
    "dense block的结构可以使feature得到更有效的利用，而且由于增加了bottleneck layers和transition layers（使用基本的growth rate）对通道数进行限制，可以使网络变窄，参数数量也降低，一定程度上减轻了过拟合的问题（当然也有dropout的功劳）\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. 运行结果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1）我最开始复现网络的时候，所有池化操作的padding没有设置参数，后来发现slim中的卷积操作，padding默认是'same'，而池化操作，padding默认是'valid'，所以实际上每次经过pooling操作（最开始的初始化处理、transition layers和最后的全局池化）得到的feature map都会比论文中提到的feature map少1个pixel，如最后一个dense block出来后得到 6x6 的feature map，而论文中是7x7。于是我在最后全局池化的时候使用了 6x6 的核进行池化，相当于做了一个与论文不一样的尝试，以此完成了第一个作业尝试，提交作业时已经跑了6个小时，运行结果如下：  \n",
    "\n",
    "2018-06-09 07:29:42.513528: I tensorflow/core/kernels/logging_ops.cc:79] eval/Recall_5[0.37109375]  \n",
    "2018-06-09 07:29:42.514041: I tensorflow/core/kernels/logging_ops.cc:79] eval/Accuracy[0.153076172]  \n",
    "INFO:tensorflow:Finished evaluation at 2018-06-09-07:29:42    \n",
    "\n",
    "作业地址：https://www.tinymind.com/executions/fckl9ks6  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2）另外建立了一个模型，池化操作增加参数padding = 'same',得到的每个阶段的feature map size和论文的一样，完成第二个作业尝试，提交作业时跑了4个小时，运行结果如下：  \n",
    "\n",
    "2018-06-09 09:47:36.289516: I tensorflow/core/kernels/logging_ops.cc:79] eval/Accuracy[0.170166016]  \n",
    "2018-06-09 09:47:36.290825: I tensorflow/core/kernels/logging_ops.cc:79] eval/Recall_5[0.385253906]  \n",
    "INFO:tensorflow:Finished evaluation at 2018-06-09-09:47:36    \n",
    "\n",
    "作业地址：https://www.tinymind.com/executions/6bz36cgd  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "发现按照原论文提出的各个参数复现网络比我修改了参数的网络效果好，跑了4个小时比第一个模型的6个小时收敛更快，而且accuracy和recall都有所提升，目前随着训练还有继续收敛趋势"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
